{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Regression with Images and Text\n",
    "\n",
    "In this notebook we will go through a series of examples on how to combine all Wide & Deep components, the Wide component (`wide`), the stack of dense layers for the \"categorical embeddings\" and numerical column (`deepdense`), the text data (`deeptext`) and images (`deepimage`). \n",
    "\n",
    "To that aim I will use the Airbnb listings dataset for London, which you can download from [here](http://insideairbnb.com/get-the-data.html). I use this dataset simply because it contains tabular data, images and text.\n",
    "\n",
    "I have taken a sample of 1000 listings to keep the data tractable in this notebook. Also, I have preprocess the data and prepared it for this excercise. All preprocessing steps can be found in the notebook `airbnb_data_preprocessing.ipynb` in this `examples` folder. Note that you do not need to go through that notebook to get an understanding on how to use the `pytorch-widedeep` library. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n",
    "import torch\n",
    "\n",
    "from pytorch_widedeep.preprocessing import WidePreprocessor, DensePreprocessor, TextPreprocessor, ImagePreprocessor\n",
    "from pytorch_widedeep.models import Wide, DeepDense, DeepText, DeepImage, WideDeep\n",
    "from pytorch_widedeep.initializers import *\n",
    "from pytorch_widedeep.callbacks import *\n",
    "from pytorch_widedeep.optim import RAdam"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>host_id</th>\n",
       "      <th>description</th>\n",
       "      <th>host_listings_count</th>\n",
       "      <th>host_identity_verified</th>\n",
       "      <th>neighbourhood_cleansed</th>\n",
       "      <th>latitude</th>\n",
       "      <th>longitude</th>\n",
       "      <th>is_location_exact</th>\n",
       "      <th>property_type</th>\n",
       "      <th>room_type</th>\n",
       "      <th>accommodates</th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>beds</th>\n",
       "      <th>guests_included</th>\n",
       "      <th>minimum_nights</th>\n",
       "      <th>instant_bookable</th>\n",
       "      <th>cancellation_policy</th>\n",
       "      <th>has_house_rules</th>\n",
       "      <th>host_gender</th>\n",
       "      <th>accommodates_catg</th>\n",
       "      <th>guests_included_catg</th>\n",
       "      <th>minimum_nights_catg</th>\n",
       "      <th>host_listings_count_catg</th>\n",
       "      <th>bathrooms_catg</th>\n",
       "      <th>bedrooms_catg</th>\n",
       "      <th>beds_catg</th>\n",
       "      <th>amenity_24-hour_check-in</th>\n",
       "      <th>amenity__toilet</th>\n",
       "      <th>amenity_accessible-height_bed</th>\n",
       "      <th>amenity_accessible-height_toilet</th>\n",
       "      <th>amenity_air_conditioning</th>\n",
       "      <th>amenity_air_purifier</th>\n",
       "      <th>amenity_alfresco_bathtub</th>\n",
       "      <th>amenity_amazon_echo</th>\n",
       "      <th>amenity_baby_bath</th>\n",
       "      <th>amenity_baby_monitor</th>\n",
       "      <th>amenity_babysitter_recommendations</th>\n",
       "      <th>amenity_balcony</th>\n",
       "      <th>amenity_bath_towel</th>\n",
       "      <th>amenity_bathroom_essentials</th>\n",
       "      <th>amenity_bathtub</th>\n",
       "      <th>amenity_bathtub_with_bath_chair</th>\n",
       "      <th>amenity_bbq_grill</th>\n",
       "      <th>amenity_beach_essentials</th>\n",
       "      <th>amenity_beach_view</th>\n",
       "      <th>amenity_beachfront</th>\n",
       "      <th>amenity_bed_linens</th>\n",
       "      <th>amenity_bedroom_comforts</th>\n",
       "      <th>...</th>\n",
       "      <th>amenity_roll-in_shower</th>\n",
       "      <th>amenity_room-darkening_shades</th>\n",
       "      <th>amenity_safety_card</th>\n",
       "      <th>amenity_sauna</th>\n",
       "      <th>amenity_self_check-in</th>\n",
       "      <th>amenity_shampoo</th>\n",
       "      <th>amenity_shared_gym</th>\n",
       "      <th>amenity_shared_hot_tub</th>\n",
       "      <th>amenity_shared_pool</th>\n",
       "      <th>amenity_shower_chair</th>\n",
       "      <th>amenity_single_level_home</th>\n",
       "      <th>amenity_ski-in_ski-out</th>\n",
       "      <th>amenity_smart_lock</th>\n",
       "      <th>amenity_smart_tv</th>\n",
       "      <th>amenity_smoke_detector</th>\n",
       "      <th>amenity_smoking_allowed</th>\n",
       "      <th>amenity_soaking_tub</th>\n",
       "      <th>amenity_sound_system</th>\n",
       "      <th>amenity_stair_gates</th>\n",
       "      <th>amenity_stand_alone_steam_shower</th>\n",
       "      <th>amenity_standing_valet</th>\n",
       "      <th>amenity_steam_oven</th>\n",
       "      <th>amenity_stove</th>\n",
       "      <th>amenity_suitable_for_events</th>\n",
       "      <th>amenity_sun_loungers</th>\n",
       "      <th>amenity_table_corner_guards</th>\n",
       "      <th>amenity_tennis_court</th>\n",
       "      <th>amenity_terrace</th>\n",
       "      <th>amenity_toilet_paper</th>\n",
       "      <th>amenity_touchless_faucets</th>\n",
       "      <th>amenity_tv</th>\n",
       "      <th>amenity_walk-in_shower</th>\n",
       "      <th>amenity_warming_drawer</th>\n",
       "      <th>amenity_washer</th>\n",
       "      <th>amenity_washer_dryer</th>\n",
       "      <th>amenity_waterfront</th>\n",
       "      <th>amenity_well-lit_path_to_entrance</th>\n",
       "      <th>amenity_wheelchair_accessible</th>\n",
       "      <th>amenity_wide_clearance_to_shower</th>\n",
       "      <th>amenity_wide_doorway_to_guest_bathroom</th>\n",
       "      <th>amenity_wide_entrance</th>\n",
       "      <th>amenity_wide_entrance_for_guests</th>\n",
       "      <th>amenity_wide_entryway</th>\n",
       "      <th>amenity_wide_hallways</th>\n",
       "      <th>amenity_wifi</th>\n",
       "      <th>amenity_window_guards</th>\n",
       "      <th>amenity_wine_cooler</th>\n",
       "      <th>security_deposit</th>\n",
       "      <th>extra_people</th>\n",
       "      <th>yield</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>13913.jpg</td>\n",
       "      <td>54730</td>\n",
       "      <td>My bright double bedroom with a large window has a relaxed feeling! It comfortably fits one or t...</td>\n",
       "      <td>4.0</td>\n",
       "      <td>f</td>\n",
       "      <td>Islington</td>\n",
       "      <td>51.56802</td>\n",
       "      <td>-0.11121</td>\n",
       "      <td>t</td>\n",
       "      <td>apartment</td>\n",
       "      <td>private_room</td>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>f</td>\n",
       "      <td>moderate</td>\n",
       "      <td>1</td>\n",
       "      <td>female</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>12.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>15400.jpg</td>\n",
       "      <td>60302</td>\n",
       "      <td>Lots of windows and light.  St Luke's Gardens are at the end of the block, and the river not too...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>t</td>\n",
       "      <td>Kensington and Chelsea</td>\n",
       "      <td>51.48796</td>\n",
       "      <td>-0.16898</td>\n",
       "      <td>t</td>\n",
       "      <td>apartment</td>\n",
       "      <td>entire_home/apt</td>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>f</td>\n",
       "      <td>strict_14_with_grace_period</td>\n",
       "      <td>1</td>\n",
       "      <td>female</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>150.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>109.50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>17402.jpg</td>\n",
       "      <td>67564</td>\n",
       "      <td>Open from June 2018 after a 3-year break, we are delighted to be welcoming guests again to this ...</td>\n",
       "      <td>19.0</td>\n",
       "      <td>t</td>\n",
       "      <td>Westminster</td>\n",
       "      <td>51.52098</td>\n",
       "      <td>-0.14002</td>\n",
       "      <td>t</td>\n",
       "      <td>apartment</td>\n",
       "      <td>entire_home/apt</td>\n",
       "      <td>6</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>t</td>\n",
       "      <td>strict_14_with_grace_period</td>\n",
       "      <td>1</td>\n",
       "      <td>female</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>350.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>149.65</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>24328.jpg</td>\n",
       "      <td>41759</td>\n",
       "      <td>Artist house, bright high ceiling rooms, private parking and a communal garden in a conservation...</td>\n",
       "      <td>2.0</td>\n",
       "      <td>t</td>\n",
       "      <td>Wandsworth</td>\n",
       "      <td>51.47298</td>\n",
       "      <td>-0.16376</td>\n",
       "      <td>t</td>\n",
       "      <td>other</td>\n",
       "      <td>entire_home/apt</td>\n",
       "      <td>2</td>\n",
       "      <td>1.5</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>30</td>\n",
       "      <td>f</td>\n",
       "      <td>moderate</td>\n",
       "      <td>1</td>\n",
       "      <td>male</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>250.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>215.60</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>25023.jpg</td>\n",
       "      <td>102813</td>\n",
       "      <td>Large, all comforts, 2-bed flat; first floor; lift; pretty communal gardens + off-street parking...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>f</td>\n",
       "      <td>Wandsworth</td>\n",
       "      <td>51.44687</td>\n",
       "      <td>-0.21874</td>\n",
       "      <td>t</td>\n",
       "      <td>apartment</td>\n",
       "      <td>entire_home/apt</td>\n",
       "      <td>4</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>f</td>\n",
       "      <td>moderate</td>\n",
       "      <td>1</td>\n",
       "      <td>female</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>250.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>79.35</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 223 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          id  host_id  \\\n",
       "0  13913.jpg    54730   \n",
       "1  15400.jpg    60302   \n",
       "2  17402.jpg    67564   \n",
       "3  24328.jpg    41759   \n",
       "4  25023.jpg   102813   \n",
       "\n",
       "                                                                                           description  \\\n",
       "0  My bright double bedroom with a large window has a relaxed feeling! It comfortably fits one or t...   \n",
       "1  Lots of windows and light.  St Luke's Gardens are at the end of the block, and the river not too...   \n",
       "2  Open from June 2018 after a 3-year break, we are delighted to be welcoming guests again to this ...   \n",
       "3  Artist house, bright high ceiling rooms, private parking and a communal garden in a conservation...   \n",
       "4  Large, all comforts, 2-bed flat; first floor; lift; pretty communal gardens + off-street parking...   \n",
       "\n",
       "   host_listings_count host_identity_verified  neighbourhood_cleansed  \\\n",
       "0                  4.0                      f               Islington   \n",
       "1                  1.0                      t  Kensington and Chelsea   \n",
       "2                 19.0                      t             Westminster   \n",
       "3                  2.0                      t              Wandsworth   \n",
       "4                  1.0                      f              Wandsworth   \n",
       "\n",
       "   latitude  longitude is_location_exact property_type        room_type  \\\n",
       "0  51.56802   -0.11121                 t     apartment     private_room   \n",
       "1  51.48796   -0.16898                 t     apartment  entire_home/apt   \n",
       "2  51.52098   -0.14002                 t     apartment  entire_home/apt   \n",
       "3  51.47298   -0.16376                 t         other  entire_home/apt   \n",
       "4  51.44687   -0.21874                 t     apartment  entire_home/apt   \n",
       "\n",
       "   accommodates  bathrooms  bedrooms  beds  guests_included  minimum_nights  \\\n",
       "0             2        1.0       1.0   0.0                1               1   \n",
       "1             2        1.0       1.0   1.0                2               3   \n",
       "2             6        2.0       3.0   3.0                4               3   \n",
       "3             2        1.5       1.0   1.0                2              30   \n",
       "4             4        1.0       2.0   2.0                2               4   \n",
       "\n",
       "  instant_bookable          cancellation_policy  has_house_rules host_gender  \\\n",
       "0                f                     moderate                1      female   \n",
       "1                f  strict_14_with_grace_period                1      female   \n",
       "2                t  strict_14_with_grace_period                1      female   \n",
       "3                f                     moderate                1        male   \n",
       "4                f                     moderate                1      female   \n",
       "\n",
       "   accommodates_catg  guests_included_catg  minimum_nights_catg  \\\n",
       "0                  2                     1                    1   \n",
       "1                  2                     2                    3   \n",
       "2                  3                     3                    3   \n",
       "3                  2                     2                    3   \n",
       "4                  3                     2                    3   \n",
       "\n",
       "   host_listings_count_catg  bathrooms_catg  bedrooms_catg  beds_catg  \\\n",
       "0                         3               1              1          0   \n",
       "1                         1               1              1          1   \n",
       "2                         3               2              3          3   \n",
       "3                         2               2              1          1   \n",
       "4                         1               1              2          2   \n",
       "\n",
       "   amenity_24-hour_check-in  amenity__toilet  amenity_accessible-height_bed  \\\n",
       "0                         0                0                              1   \n",
       "1                         1                0                              0   \n",
       "2                         0                0                              0   \n",
       "3                         1                0                              0   \n",
       "4                         0                0                              0   \n",
       "\n",
       "   amenity_accessible-height_toilet  amenity_air_conditioning  \\\n",
       "0                                 1                         0   \n",
       "1                                 0                         1   \n",
       "2                                 0                         0   \n",
       "3                                 0                         0   \n",
       "4                                 0                         0   \n",
       "\n",
       "   amenity_air_purifier  amenity_alfresco_bathtub  amenity_amazon_echo  \\\n",
       "0                     0                         0                    0   \n",
       "1                     0                         0                    0   \n",
       "2                     0                         0                    0   \n",
       "3                     0                         0                    0   \n",
       "4                     0                         0                    0   \n",
       "\n",
       "   amenity_baby_bath  amenity_baby_monitor  \\\n",
       "0                  0                     0   \n",
       "1                  0                     0   \n",
       "2                  0                     0   \n",
       "3                  0                     0   \n",
       "4                  0                     0   \n",
       "\n",
       "   amenity_babysitter_recommendations  amenity_balcony  amenity_bath_towel  \\\n",
       "0                                   1                0                   0   \n",
       "1                                   0                0                   0   \n",
       "2                                   0                0                   0   \n",
       "3                                   0                0                   0   \n",
       "4                                   0                0                   0   \n",
       "\n",
       "   amenity_bathroom_essentials  amenity_bathtub  \\\n",
       "0                            0                1   \n",
       "1                            0                0   \n",
       "2                            0                0   \n",
       "3                            0                0   \n",
       "4                            0                0   \n",
       "\n",
       "   amenity_bathtub_with_bath_chair  amenity_bbq_grill  \\\n",
       "0                                1                  0   \n",
       "1                                0                  0   \n",
       "2                                0                  0   \n",
       "3                                0                  0   \n",
       "4                                0                  0   \n",
       "\n",
       "   amenity_beach_essentials  amenity_beach_view  amenity_beachfront  \\\n",
       "0                         0                   0                   0   \n",
       "1                         0                   0                   0   \n",
       "2                         0                   0                   0   \n",
       "3                         0                   0                   0   \n",
       "4                         0                   0                   0   \n",
       "\n",
       "   amenity_bed_linens  amenity_bedroom_comforts  ...  amenity_roll-in_shower  \\\n",
       "0                   1                         0  ...                       1   \n",
       "1                   0                         0  ...                       0   \n",
       "2                   1                         0  ...                       0   \n",
       "3                   0                         0  ...                       0   \n",
       "4                   0                         0  ...                       0   \n",
       "\n",
       "   amenity_room-darkening_shades  amenity_safety_card  amenity_sauna  \\\n",
       "0                              1                    0              0   \n",
       "1                              0                    0              0   \n",
       "2                              0                    0              0   \n",
       "3                              0                    0              0   \n",
       "4                              0                    0              0   \n",
       "\n",
       "   amenity_self_check-in  amenity_shampoo  amenity_shared_gym  \\\n",
       "0                      0                1                   0   \n",
       "1                      0                1                   0   \n",
       "2                      1                1                   0   \n",
       "3                      1                1                   0   \n",
       "4                      0                0                   0   \n",
       "\n",
       "   amenity_shared_hot_tub  amenity_shared_pool  amenity_shower_chair  \\\n",
       "0                       0                    0                     0   \n",
       "1                       0                    0                     0   \n",
       "2                       0                    0                     0   \n",
       "3                       0                    0                     0   \n",
       "4                       0                    0                     0   \n",
       "\n",
       "   amenity_single_level_home  amenity_ski-in_ski-out  amenity_smart_lock  \\\n",
       "0                          0                       0                   0   \n",
       "1                          0                       0                   0   \n",
       "2                          0                       0                   0   \n",
       "3                          0                       0                   0   \n",
       "4                          0                       0                   0   \n",
       "\n",
       "   amenity_smart_tv  amenity_smoke_detector  amenity_smoking_allowed  \\\n",
       "0                 0                       1                        1   \n",
       "1                 0                       1                        0   \n",
       "2                 0                       1                        0   \n",
       "3                 0                       1                        0   \n",
       "4                 0                       1                        0   \n",
       "\n",
       "   amenity_soaking_tub  amenity_sound_system  amenity_stair_gates  \\\n",
       "0                    0                     0                    0   \n",
       "1                    0                     0                    0   \n",
       "2                    0                     0                    0   \n",
       "3                    0                     0                    0   \n",
       "4                    0                     0                    0   \n",
       "\n",
       "   amenity_stand_alone_steam_shower  amenity_standing_valet  \\\n",
       "0                                 0                       0   \n",
       "1                                 0                       0   \n",
       "2                                 0                       0   \n",
       "3                                 0                       0   \n",
       "4                                 0                       0   \n",
       "\n",
       "   amenity_steam_oven  amenity_stove  amenity_suitable_for_events  \\\n",
       "0                   0              1                            0   \n",
       "1                   0              0                            0   \n",
       "2                   0              1                            0   \n",
       "3                   0              0                            0   \n",
       "4                   0              0                            0   \n",
       "\n",
       "   amenity_sun_loungers  amenity_table_corner_guards  amenity_tennis_court  \\\n",
       "0                     0                            0                     0   \n",
       "1                     0                            0                     0   \n",
       "2                     0                            0                     0   \n",
       "3                     0                            0                     0   \n",
       "4                     0                            0                     0   \n",
       "\n",
       "   amenity_terrace  amenity_toilet_paper  amenity_touchless_faucets  \\\n",
       "0                0                     0                          0   \n",
       "1                0                     0                          0   \n",
       "2                0                     0                          0   \n",
       "3                0                     0                          0   \n",
       "4                0                     0                          0   \n",
       "\n",
       "   amenity_tv  amenity_walk-in_shower  amenity_warming_drawer  amenity_washer  \\\n",
       "0           1                       0                       0               1   \n",
       "1           1                       0                       0               1   \n",
       "2           1                       0                       0               1   \n",
       "3           1                       0                       0               1   \n",
       "4           1                       0                       0               1   \n",
       "\n",
       "   amenity_washer_dryer  amenity_waterfront  \\\n",
       "0                     0                   0   \n",
       "1                     0                   0   \n",
       "2                     0                   0   \n",
       "3                     0                   0   \n",
       "4                     0                   0   \n",
       "\n",
       "   amenity_well-lit_path_to_entrance  amenity_wheelchair_accessible  \\\n",
       "0                                  0                              0   \n",
       "1                                  0                              0   \n",
       "2                                  0                              0   \n",
       "3                                  0                              0   \n",
       "4                                  0                              0   \n",
       "\n",
       "   amenity_wide_clearance_to_shower  amenity_wide_doorway_to_guest_bathroom  \\\n",
       "0                                 0                                       1   \n",
       "1                                 0                                       0   \n",
       "2                                 0                                       0   \n",
       "3                                 0                                       0   \n",
       "4                                 0                                       0   \n",
       "\n",
       "   amenity_wide_entrance  amenity_wide_entrance_for_guests  \\\n",
       "0                      1                                 0   \n",
       "1                      0                                 0   \n",
       "2                      0                                 0   \n",
       "3                      0                                 0   \n",
       "4                      0                                 0   \n",
       "\n",
       "   amenity_wide_entryway  amenity_wide_hallways  amenity_wifi  \\\n",
       "0                      0                      0             1   \n",
       "1                      0                      0             1   \n",
       "2                      0                      0             1   \n",
       "3                      0                      0             1   \n",
       "4                      0                      0             1   \n",
       "\n",
       "   amenity_window_guards  amenity_wine_cooler  security_deposit  extra_people  \\\n",
       "0                      0                    0             100.0          15.0   \n",
       "1                      0                    0             150.0           0.0   \n",
       "2                      0                    0             350.0          10.0   \n",
       "3                      0                    0             250.0           0.0   \n",
       "4                      0                    0             250.0          11.0   \n",
       "\n",
       "    yield  \n",
       "0   12.00  \n",
       "1  109.50  \n",
       "2  149.65  \n",
       "3  215.60  \n",
       "4   79.35  \n",
       "\n",
       "[5 rows x 223 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('data/airbnb/airbnb_sample.csv')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Regression with the defaults\n",
    "\n",
    "The set up"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# There are a number of columns that are already binary. Therefore, no need to one hot encode them\n",
    "crossed_cols = (['property_type', 'room_type'],)\n",
    "already_dummies = [c for c in df.columns if 'amenity' in c] + ['has_house_rules']\n",
    "wide_cols = ['is_location_exact', 'property_type', 'room_type', 'host_gender',\n",
    "'instant_bookable'] + already_dummies\n",
    "cat_embed_cols = [(c, 16) for c in df.columns if 'catg' in c] + \\\n",
    "    [('neighbourhood_cleansed', 64), ('cancellation_policy', 16)]\n",
    "continuous_cols = ['latitude', 'longitude', 'security_deposit', 'extra_people']\n",
    "# it does not make sense to standarised Latitude and Longitude\n",
    "already_standard = ['latitude', 'longitude']\n",
    "# text and image colnames\n",
    "text_col = 'description'\n",
    "img_col = 'id'\n",
    "# path to pretrained word embeddings and the images\n",
    "word_vectors_path = 'data/glove.6B/glove.6B.100d.txt'\n",
    "img_path = 'data/airbnb/property_picture'\n",
    "# target\n",
    "target_col = 'yield'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prepare the data\n",
    "\n",
    "I will focus here on how to prepare the data and run the model. Check notebooks 1 and 2 to see what's going on behind the scences\n",
    "\n",
    "Preparing the data is rather simple"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "target = df[target_col].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\n",
    "X_wide = wide_preprocessor.fit_transform(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "deep_preprocessor = DensePreprocessor(embed_cols=cat_embed_cols, continuous_cols=continuous_cols)\n",
    "X_deep = deep_preprocessor.fit_transform(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The vocabulary contains 2192 tokens\n",
      "Indexing word vectors...\n",
      "Loaded 400000 word vectors\n",
      "Preparing embeddings matrix...\n",
      "2175 words in the vocabulary had data/glove.6B/glove.6B.100d.txt vectors and appear more than 5 times\n"
     ]
    }
   ],
   "source": [
    "text_preprocessor = TextPreprocessor(word_vectors_path=word_vectors_path, text_col=text_col)\n",
    "X_text = text_preprocessor.fit_transform(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading Images from data/airbnb/property_picture\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  4%|▍         | 43/1001 [00:00<00:02, 424.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Resizing\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1001/1001 [00:02<00:00, 400.65it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing normalisation metrics\n"
     ]
    }
   ],
   "source": [
    "image_processor = ImagePreprocessor(img_col = img_col, img_path = img_path)\n",
    "X_images = image_processor.fit_transform(df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Build the model components"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Linear model\n",
    "wide = Wide(wide_dim=np.unique(X_wide).shape[0], pred_dim=1)\n",
    "# DeepDense: 2 Dense layers\n",
    "deepdense = DeepDense(hidden_layers=[128,64], dropout=[0.5, 0.5], \n",
    "                      deep_column_idx=deep_preprocessor.deep_column_idx,\n",
    "                      embed_input=deep_preprocessor.embeddings_input,\n",
    "                      continuous_cols=continuous_cols)\n",
    "# DeepText: a stack of 2 LSTMs\n",
    "deeptext = DeepText(vocab_size=len(text_preprocessor.vocab.itos), hidden_dim=64, \n",
    "                    n_layers=2, rnn_dropout=0.5, \n",
    "                    embedding_matrix=text_preprocessor.embedding_matrix)\n",
    "# Pretrained Resnet 18 (default is all but last 2 conv blocks frozen) plus a FC-Head 512->256->128\n",
    "deepimage = DeepImage(pretrained=True, head_layers=[512, 256, 128])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Combine them all with the \"collector\" class `WideDeep`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = WideDeep(wide=wide, deepdense=deepdense, deeptext=deeptext, deepimage=deepimage)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compile and fit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(method='regression')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/25 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "epoch 1: 100%|██████████| 25/25 [01:08<00:00,  2.74s/it, loss=1.73e+4]\n",
      "valid: 100%|██████████| 7/7 [00:14<00:00,  2.01s/it, loss=1.45e+4]\n"
     ]
    }
   ],
   "source": [
    "model.fit(X_wide=X_wide, X_deep=X_deep, X_text=X_text, X_img=X_images,\n",
    "    target=target, n_epochs=1, batch_size=32, val_split=0.2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Regression with varying parameters and a fully connected head (FC-Head) receiving the full deep side\n",
    "\n",
    "This would be the second architecture shown in the README file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "wide = Wide(wide_dim=np.unique(X_wide).shape[0], pred_dim=1)\n",
    "deepdense = DeepDense(hidden_layers=[128,64], dropout=[0.5, 0.5], \n",
    "                      deep_column_idx=deep_preprocessor.deep_column_idx,\n",
    "                      embed_input=deep_preprocessor.embeddings_input,\n",
    "                      continuous_cols=continuous_cols)\n",
    "deeptext = DeepText(vocab_size=len(text_preprocessor.vocab.itos), hidden_dim=128, \n",
    "                    n_layers=2, rnn_dropout=0.5, \n",
    "                    embedding_matrix=text_preprocessor.embedding_matrix)\n",
    "deepimage = DeepImage(pretrained=True, head_layers=[512, 256, 128])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The **FC-Head** is passed as a parameter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = WideDeep(wide=wide, deepdense=deepdense, deeptext=deeptext, deepimage=deepimage, head_layers=[128, 64])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's have a look to the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "WideDeep(\n",
       "  (wide): Wide(\n",
       "    (wide_linear): Embedding(357, 1, padding_idx=0)\n",
       "  )\n",
       "  (deepdense): DeepDense(\n",
       "    (embed_layers): ModuleDict(\n",
       "      (emb_layer_accommodates_catg): Embedding(4, 16)\n",
       "      (emb_layer_bathrooms_catg): Embedding(4, 16)\n",
       "      (emb_layer_bedrooms_catg): Embedding(5, 16)\n",
       "      (emb_layer_beds_catg): Embedding(5, 16)\n",
       "      (emb_layer_cancellation_policy): Embedding(6, 16)\n",
       "      (emb_layer_guests_included_catg): Embedding(4, 16)\n",
       "      (emb_layer_host_listings_count_catg): Embedding(5, 16)\n",
       "      (emb_layer_minimum_nights_catg): Embedding(4, 16)\n",
       "      (emb_layer_neighbourhood_cleansed): Embedding(33, 64)\n",
       "    )\n",
       "    (embed_dropout): Dropout(p=0.0, inplace=False)\n",
       "    (dense): Sequential(\n",
       "      (dense_layer_0): Sequential(\n",
       "        (0): Linear(in_features=196, out_features=128, bias=True)\n",
       "        (1): LeakyReLU(negative_slope=0.01, inplace=True)\n",
       "        (2): Dropout(p=0.5, inplace=False)\n",
       "      )\n",
       "      (dense_layer_1): Sequential(\n",
       "        (0): Linear(in_features=128, out_features=64, bias=True)\n",
       "        (1): LeakyReLU(negative_slope=0.01, inplace=True)\n",
       "        (2): Dropout(p=0.5, inplace=False)\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (deeptext): DeepText(\n",
       "    (word_embed): Embedding(2192, 100, padding_idx=1)\n",
       "    (rnn): LSTM(100, 128, num_layers=2, batch_first=True, dropout=0.5)\n",
       "  )\n",
       "  (deepimage): DeepImage(\n",
       "    (backbone): Sequential(\n",
       "      (0): Conv2d(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)\n",
       "      (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (2): ReLU(inplace=True)\n",
       "      (3): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)\n",
       "      (4): Sequential(\n",
       "        (0): BasicBlock(\n",
       "          (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "          (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (relu): ReLU(inplace=True)\n",
       "          (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "          (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "        (1): BasicBlock(\n",
       "          (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "          (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (relu): ReLU(inplace=True)\n",
       "          (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "          (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (5): Sequential(\n",
       "        (0): BasicBlock(\n",
       "          (conv1): Conv2d(64, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "          (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (relu): ReLU(inplace=True)\n",
       "          (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "          (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (downsample): Sequential(\n",
       "            (0): Conv2d(64, 128, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
       "            (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          )\n",
       "        )\n",
       "        (1): BasicBlock(\n",
       "          (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "          (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (relu): ReLU(inplace=True)\n",
       "          (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "          (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (6): Sequential(\n",
       "        (0): BasicBlock(\n",
       "          (conv1): Conv2d(128, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "          (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (relu): ReLU(inplace=True)\n",
       "          (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "          (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (downsample): Sequential(\n",
       "            (0): Conv2d(128, 256, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
       "            (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          )\n",
       "        )\n",
       "        (1): BasicBlock(\n",
       "          (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "          (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (relu): ReLU(inplace=True)\n",
       "          (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "          (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (7): Sequential(\n",
       "        (0): BasicBlock(\n",
       "          (conv1): Conv2d(256, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "          (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (relu): ReLU(inplace=True)\n",
       "          (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "          (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (downsample): Sequential(\n",
       "            (0): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
       "            (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          )\n",
       "        )\n",
       "        (1): BasicBlock(\n",
       "          (conv1): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "          (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (relu): ReLU(inplace=True)\n",
       "          (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "          (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (8): AdaptiveAvgPool2d(output_size=(1, 1))\n",
       "    )\n",
       "    (imagehead): Sequential(\n",
       "      (dense_layer_0): Sequential(\n",
       "        (0): Linear(in_features=512, out_features=256, bias=True)\n",
       "        (1): LeakyReLU(negative_slope=0.01, inplace=True)\n",
       "        (2): Dropout(p=0.0, inplace=False)\n",
       "      )\n",
       "      (dense_layer_1): Sequential(\n",
       "        (0): Linear(in_features=256, out_features=128, bias=True)\n",
       "        (1): LeakyReLU(negative_slope=0.01, inplace=True)\n",
       "        (2): Dropout(p=0.0, inplace=False)\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (deephead): Sequential(\n",
       "    (head_layer_0): Sequential(\n",
       "      (0): Linear(in_features=320, out_features=128, bias=True)\n",
       "      (1): LeakyReLU(negative_slope=0.01, inplace=True)\n",
       "      (2): Dropout(p=0.0, inplace=False)\n",
       "    )\n",
       "    (head_layer_1): Sequential(\n",
       "      (0): Linear(in_features=128, out_features=64, bias=True)\n",
       "      (1): LeakyReLU(negative_slope=0.01, inplace=True)\n",
       "      (2): Dropout(p=0.0, inplace=False)\n",
       "    )\n",
       "    (head_out): Linear(in_features=64, out_features=1, bias=True)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Both, the Text and Image components allow FC-heads on their own (referred very creatively as `texthead` and `imagehead`). Following this nomenclature, the FC-head that receives the concatenation of the whole deep component is called `deephead`. \n",
    "\n",
    "Now let's go \"kaggle crazy\". Let's use different optimizers, initializers and schedulers for different components. Moreover, let's use a different learning rate for different parameter groups, for the `DeepDense` component"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "deep_params = []\n",
    "for childname, child in model.named_children():\n",
    "    if childname == 'deepdense':\n",
    "        for n,p in child.named_parameters():\n",
    "            if \"emb_layer\" in n: deep_params.append({'params': p, 'lr': 1e-4})\n",
    "            else: deep_params.append({'params': p, 'lr': 1e-3})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "wide_opt = torch.optim.Adam(model.wide.parameters(), lr=0.03)\n",
    "deep_opt = torch.optim.Adam(deep_params)\n",
    "text_opt = RAdam(model.deeptext.parameters())\n",
    "img_opt  = RAdam(model.deepimage.parameters())\n",
    "head_opt = torch.optim.Adam(model.deephead.parameters())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "wide_sch = torch.optim.lr_scheduler.StepLR(wide_opt, step_size=5)\n",
    "deep_sch = torch.optim.lr_scheduler.MultiStepLR(deep_opt, milestones=[3,8])\n",
    "text_sch = torch.optim.lr_scheduler.StepLR(text_opt, step_size=5)\n",
    "img_sch  = torch.optim.lr_scheduler.MultiStepLR(deep_opt, milestones=[3,8])\n",
    "head_sch = torch.optim.lr_scheduler.StepLR(head_opt, step_size=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# remember, one optimizer per model components, for lr_schedures and initializers is not neccesary\n",
    "optimizers = {'wide': wide_opt, 'deepdense':deep_opt, 'deeptext':text_opt, 'deepimage': img_opt, 'deephead': head_opt}\n",
    "schedulers = {'wide': wide_sch, 'deepdense':deep_sch, 'deeptext':text_sch, 'deepimage': img_sch, 'deephead': head_sch}\n",
    "\n",
    "# Now...we have used pretrained word embeddings, so you do not want to\n",
    "# initialise these  embeddings. However you might still want to initialise the\n",
    "# other layers in the DeepText component. No probs, you can do that with the\n",
    "# parameter pattern and your knowledge on regular  expressions. Here we are\n",
    "# telling to the KaimingNormal initializer to NOT touch the  parameters whose\n",
    "# name contains the string word_embed. \n",
    "initializers = {'wide': KaimingNormal, 'deepdense':KaimingNormal, \n",
    "                'deeptext':KaimingNormal(pattern=r\"^(?!.*word_embed).*$\"), \n",
    "                'deepimage':KaimingNormal}\n",
    "\n",
    "mean = [0.406, 0.456, 0.485]  #BGR\n",
    "std =  [0.225, 0.224, 0.229]  #BGR\n",
    "transforms = [ToTensor, Normalize(mean=mean, std=std)]\n",
    "callbacks = [LRHistory(n_epochs=10), EarlyStopping, ModelCheckpoint(filepath='model_weights/wd_out')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(method='regression', initializers=initializers, optimizers=optimizers,\n",
    "    lr_schedulers=schedulers, callbacks=callbacks, transforms=transforms)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "WideDeep(\n",
       "  (wide): Wide(\n",
       "    (wide_linear): Embedding(357, 1, padding_idx=0)\n",
       "  )\n",
       "  (deepdense): DeepDense(\n",
       "    (embed_layers): ModuleDict(\n",
       "      (emb_layer_accommodates_catg): Embedding(4, 16)\n",
       "      (emb_layer_bathrooms_catg): Embedding(4, 16)\n",
       "      (emb_layer_bedrooms_catg): Embedding(5, 16)\n",
       "      (emb_layer_beds_catg): Embedding(5, 16)\n",
       "      (emb_layer_cancellation_policy): Embedding(6, 16)\n",
       "      (emb_layer_guests_included_catg): Embedding(4, 16)\n",
       "      (emb_layer_host_listings_count_catg): Embedding(5, 16)\n",
       "      (emb_layer_minimum_nights_catg): Embedding(4, 16)\n",
       "      (emb_layer_neighbourhood_cleansed): Embedding(33, 64)\n",
       "    )\n",
       "    (embed_dropout): Dropout(p=0.0, inplace=False)\n",
       "    (dense): Sequential(\n",
       "      (dense_layer_0): Sequential(\n",
       "        (0): Linear(in_features=196, out_features=128, bias=True)\n",
       "        (1): LeakyReLU(negative_slope=0.01, inplace=True)\n",
       "        (2): Dropout(p=0.5, inplace=False)\n",
       "      )\n",
       "      (dense_layer_1): Sequential(\n",
       "        (0): Linear(in_features=128, out_features=64, bias=True)\n",
       "        (1): LeakyReLU(negative_slope=0.01, inplace=True)\n",
       "        (2): Dropout(p=0.5, inplace=False)\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (deeptext): DeepText(\n",
       "    (word_embed): Embedding(2192, 100, padding_idx=1)\n",
       "    (rnn): LSTM(100, 128, num_layers=2, batch_first=True, dropout=0.5)\n",
       "  )\n",
       "  (deepimage): DeepImage(\n",
       "    (backbone): Sequential(\n",
       "      (0): Conv2d(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)\n",
       "      (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (2): ReLU(inplace=True)\n",
       "      (3): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)\n",
       "      (4): Sequential(\n",
       "        (0): BasicBlock(\n",
       "          (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "          (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (relu): ReLU(inplace=True)\n",
       "          (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "          (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "        (1): BasicBlock(\n",
       "          (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "          (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (relu): ReLU(inplace=True)\n",
       "          (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "          (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (5): Sequential(\n",
       "        (0): BasicBlock(\n",
       "          (conv1): Conv2d(64, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "          (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (relu): ReLU(inplace=True)\n",
       "          (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "          (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (downsample): Sequential(\n",
       "            (0): Conv2d(64, 128, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
       "            (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          )\n",
       "        )\n",
       "        (1): BasicBlock(\n",
       "          (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "          (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (relu): ReLU(inplace=True)\n",
       "          (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "          (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (6): Sequential(\n",
       "        (0): BasicBlock(\n",
       "          (conv1): Conv2d(128, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "          (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (relu): ReLU(inplace=True)\n",
       "          (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "          (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (downsample): Sequential(\n",
       "            (0): Conv2d(128, 256, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
       "            (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          )\n",
       "        )\n",
       "        (1): BasicBlock(\n",
       "          (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "          (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (relu): ReLU(inplace=True)\n",
       "          (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "          (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (7): Sequential(\n",
       "        (0): BasicBlock(\n",
       "          (conv1): Conv2d(256, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "          (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (relu): ReLU(inplace=True)\n",
       "          (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "          (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (downsample): Sequential(\n",
       "            (0): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
       "            (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          )\n",
       "        )\n",
       "        (1): BasicBlock(\n",
       "          (conv1): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "          (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (relu): ReLU(inplace=True)\n",
       "          (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "          (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (8): AdaptiveAvgPool2d(output_size=(1, 1))\n",
       "    )\n",
       "    (imagehead): Sequential(\n",
       "      (dense_layer_0): Sequential(\n",
       "        (0): Linear(in_features=512, out_features=256, bias=True)\n",
       "        (1): LeakyReLU(negative_slope=0.01, inplace=True)\n",
       "        (2): Dropout(p=0.0, inplace=False)\n",
       "      )\n",
       "      (dense_layer_1): Sequential(\n",
       "        (0): Linear(in_features=256, out_features=128, bias=True)\n",
       "        (1): LeakyReLU(negative_slope=0.01, inplace=True)\n",
       "        (2): Dropout(p=0.0, inplace=False)\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (deephead): Sequential(\n",
       "    (head_layer_0): Sequential(\n",
       "      (0): Linear(in_features=320, out_features=128, bias=True)\n",
       "      (1): LeakyReLU(negative_slope=0.01, inplace=True)\n",
       "      (2): Dropout(p=0.0, inplace=False)\n",
       "    )\n",
       "    (head_layer_1): Sequential(\n",
       "      (0): Linear(in_features=128, out_features=64, bias=True)\n",
       "      (1): LeakyReLU(negative_slope=0.01, inplace=True)\n",
       "      (2): Dropout(p=0.0, inplace=False)\n",
       "    )\n",
       "    (head_out): Linear(in_features=64, out_features=1, bias=True)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/25 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "epoch 1: 100%|██████████| 25/25 [02:04<00:00,  4.98s/it, loss=1.24e+4]\n",
      "valid: 100%|██████████| 7/7 [00:16<00:00,  2.33s/it, loss=9.26e+3]\n"
     ]
    }
   ],
   "source": [
    "model.fit(X_wide=X_wide, X_deep=X_deep, X_text=X_text, X_img=X_images,\n",
    "    target=target, n_epochs=1, batch_size=32, val_split=0.2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "we have only run one epoch, but let's check that the LRHistory callback records the lr values for each group"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'lr_wide_0': [0.03, 0.03],\n",
       " 'lr_deepdense_0': [0.0001, 0.0001],\n",
       " 'lr_deepdense_1': [0.0001, 0.0001],\n",
       " 'lr_deepdense_2': [0.0001, 0.0001],\n",
       " 'lr_deepdense_3': [0.0001, 0.0001],\n",
       " 'lr_deepdense_4': [0.0001, 0.0001],\n",
       " 'lr_deepdense_5': [0.0001, 0.0001],\n",
       " 'lr_deepdense_6': [0.0001, 0.0001],\n",
       " 'lr_deepdense_7': [0.0001, 0.0001],\n",
       " 'lr_deepdense_8': [0.0001, 0.0001],\n",
       " 'lr_deepdense_9': [0.001, 0.001],\n",
       " 'lr_deepdense_10': [0.001, 0.001],\n",
       " 'lr_deepdense_11': [0.001, 0.001],\n",
       " 'lr_deepdense_12': [0.001, 0.001],\n",
       " 'lr_deeptext_0': [0.001, 0.001],\n",
       " 'lr_deepimage_0': [0.001, 0.001],\n",
       " 'lr_deephead_0': [0.001, 0.001]}"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.lr_history"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
